import os
import requests
from markdownify import markdownify as md

def crawl(url: str, return_format: str = "html") -> str:
    headers = {
        "Content-Type": "application/json",
        "X-Return-Format": return_format,
    }
    if os.getenv("JINA_API_KEY"):
        headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
    else:
        print(
            "Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information."
        )
    data = {"url": url}
    response = requests.post("https://r.jina.ai/", headers=headers, json=data)
    return response.text


def to_markdown(html_content) -> str:
    markdown = ""
    markdown += md(html_content)
    return markdown

# 新增的调用代码
if __name__ == "__main__":
    test_url = "https://qwenlm.github.io/zh/blog/qwen3/"
    result = crawl(test_url)
    md = to_markdown(result)
    print(md)